{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.layers import Dense, Input, Flatten, concatenate, Conv1D\n",
    "from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D,Bidirectional, Convolution1D, Embedding, SpatialDropout1D, BatchNormalization,MaxPooling1D, Dropout, LSTM, GRU\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = '../input/'\n",
    "CACHE_PATH = '../cache/'\n",
    "OUTPUT_PATH ='../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = np.load(CACHE_PATH + 'data.npz')\n",
    "X_train = data['X_train']\n",
    "y_train = data['y_train']\n",
    "X_val = data['X_val']\n",
    "y_val = data['y_val']\n",
    "X_test = data['X_test']\n",
    "embedding_matrix = np.load(CACHE_PATH + 'embedding_matrix.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_FEATURES = 20000\n",
    "MAX_SEQUENCE_LENGTH = 300\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_FEATURES,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_gru_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = SpatialDropout1D(0.2)(embedded_sequences)\n",
    "    x = Bidirectional(GRU(250, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    x = Conv1D(64, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    x = concatenate([avg_pool, max_pool])\n",
    "    x = Dense(100, activation=\"relu\")(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(1, activation=\"linear\")(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_gru_model(model):\n",
    "    model_path = CACHE_PATH + \"gru_weights_best.hdf5\"\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_val, y_val), callbacks=callbacks_list)\n",
    "    model.load_weights(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_gru_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 256)     5120000     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 500)     760500      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 98, 64)       96064       bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_1 (Glo (None, 64)           0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 64)           0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 128)          0           global_average_pooling1d_1[0][0] \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 100)          12900       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 100)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            101         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 5,989,565\n",
      "Trainable params: 869,565\n",
      "Non-trainable params: 5,120,000\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model,to_file='GRU.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 258339 samples, validate on 13597 samples\n",
      "Epoch 1/100\n",
      "258339/258339 [==============================] - 346s 1ms/step - loss: 0.9110 - val_loss: 0.5956\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.59556, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 2/100\n",
      "258339/258339 [==============================] - 350s 1ms/step - loss: 0.6064 - val_loss: 0.4444\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.59556 to 0.44443, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 3/100\n",
      "258339/258339 [==============================] - 344s 1ms/step - loss: 0.5025 - val_loss: 0.3971\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.44443 to 0.39707, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 4/100\n",
      "258339/258339 [==============================] - 342s 1ms/step - loss: 0.4610 - val_loss: 0.3854\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.39707 to 0.38540, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 5/100\n",
      "258339/258339 [==============================] - 342s 1ms/step - loss: 0.4343 - val_loss: 0.4049\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "258339/258339 [==============================] - 343s 1ms/step - loss: 0.4155 - val_loss: 0.3772\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.38540 to 0.37719, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 7/100\n",
      "258339/258339 [==============================] - 345s 1ms/step - loss: 0.4025 - val_loss: 0.4082\n",
      "\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 8/100\n",
      "258339/258339 [==============================] - 345s 1ms/step - loss: 0.3910 - val_loss: 0.3594\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.37719 to 0.35942, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 9/100\n",
      "258339/258339 [==============================] - 347s 1ms/step - loss: 0.3833 - val_loss: 0.3586\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.35942 to 0.35862, saving model to ../cache/gru_weights_best.hdf5\n",
      "Epoch 10/100\n",
      "258339/258339 [==============================] - 340s 1ms/step - loss: 0.3733 - val_loss: 0.3752\n",
      "\n",
      "Epoch 00010: val_loss did not improve\n",
      "Epoch 11/100\n",
      "258339/258339 [==============================] - 336s 1ms/step - loss: 0.3674 - val_loss: 0.3968\n",
      "\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 12/100\n",
      "258339/258339 [==============================] - 335s 1ms/step - loss: 0.3619 - val_loss: 0.3737\n",
      "\n",
      "Epoch 00012: val_loss did not improve\n",
      "Epoch 13/100\n",
      "258339/258339 [==============================] - 346s 1ms/step - loss: 0.3585 - val_loss: 0.3661\n",
      "\n",
      "Epoch 00013: val_loss did not improve\n",
      "Epoch 14/100\n",
      "258339/258339 [==============================] - 346s 1ms/step - loss: 0.3551 - val_loss: 0.3694\n",
      "\n",
      "Epoch 00014: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "model = train_gru_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 20s 400us/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(X_test,batch_size=128,verbose=1)\n",
    "y_test[y_test < 1] = 1\n",
    "y_test[y_test > 4.7] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "time = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv(INPUT_PATH + 'sample.csv',header=None,names=['Id','Score'])\n",
    "sub['Score'] = y_test\n",
    "sub.to_csv(OUTPUT_PATH + 'gru_{}.csv'.format(time.strftime('%Y-%m-%d-%H:%M:%S')),index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
